{
 "cells": [
  {
   "metadata": {},
   "cell_type": "raw",
   "source": [
    "模型评估\n",
    "下载评估函数项目\n",
    "https://github.com/huggingface/evaluate/blob/main/metrics/accuracy/README.md"
   ],
   "id": "65d5df785419bfa1"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-12T03:56:16.933191Z",
     "start_time": "2024-12-12T03:56:07.229315Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import evaluate\n",
    "\n",
    "# 显示支持的评估函数\n",
    "evaluate.list_evaluation_modules(with_details=True)\n",
    "# with_details=True 看细节\n",
    "# evaluate.list_evaluation_modules(module_type=\"comparison\",\n",
    "#                                  include_community=False,\n",
    "#                                  with_details=True)"
   ],
   "id": "ea0cf7a212c3aa37",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'Remeris/rouge_ru', 'type': 'metric', 'community': True, 'likes': 1},\n",
       " {'name': 'lvwerra/test', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'jordyvl/ece', 'type': 'metric', 'community': True, 'likes': 3},\n",
       " {'name': 'angelina-wang/directional_bias_amplification',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 6},\n",
       " {'name': 'cpllab/syntaxgym', 'type': 'metric', 'community': True, 'likes': 1},\n",
       " {'name': 'lvwerra/bary_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'hack/test_metric', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'yzha/ctc_eval', 'type': 'metric', 'community': True, 'likes': 1},\n",
       " {'name': 'codeparrot/apps_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 6},\n",
       " {'name': 'mfumanelli/geometric_mean',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'daiyizheng/valid', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'erntkn/dice_coefficient',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'mgfrantz/roc_auc_macro',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Vlasta/pr_auc', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'gorkaartola/metric_for_tp_fp_samples',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'idsedykh/metric', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'idsedykh/codebleu2',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'idsedykh/codebleu',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'idsedykh/megaglue',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Vertaix/vendiscore',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 7},\n",
       " {'name': 'GMFTBY/dailydialogevaluate',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'GMFTBY/dailydialog_evaluate',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'jzm-mailchimp/joshs_second_test_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ola13/precision_at_k',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'yulong-me/yl_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'abidlabs/mean_iou',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'abidlabs/mean_iou2',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'KevinSpaghetti/accuracyk',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'NimaBoscarino/weat',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ronaldahmed/nwentfaithfulness',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Viona/infolm', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'kyokote/my_metric2',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'kashif/mape', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'Ochiroo/rouge_mn', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'giulio98/code_eval_outputs',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'leslyarun/fbeta_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'giulio98/codebleu',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'anz2/iliauniiccocrevaluation',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'zbeloki/m2', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'xu1998hz/sescore',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 15},\n",
       " {'name': 'dvitel/codebleu', 'type': 'metric', 'community': True, 'likes': 3},\n",
       " {'name': 'NCSOFT/harim_plus',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 10},\n",
       " {'name': 'JP-SystemsX/nDCG', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'sportlosos/sescore',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Drunper/metrica_tesi',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'jpxkqx/peak_signal_to_noise_ratio',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'jpxkqx/signal_to_reconstruction_error',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'hpi-dhc/FairEval', 'type': 'metric', 'community': True, 'likes': 4},\n",
       " {'name': 'lvwerra/accuracy_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ybelkada/cocoevaluate',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 3},\n",
       " {'name': 'harshhpareek/bertscore',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'posicube/mean_reciprocal_rank',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'bstrai/classification_report',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 2},\n",
       " {'name': 'omidf/squad_precision_recall',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Josh98/nl2bash_m', 'type': 'metric', 'community': True, 'likes': 1},\n",
       " {'name': 'BucketHeadP65/confusion_matrix',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 2},\n",
       " {'name': 'BucketHeadP65/roc_curve',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'yonting/average_precision_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'transZ/test_parascore',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'transZ/sbert_cosine',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'hynky/sklearn_proxy',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'xu1998hz/sescore_english_mt',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'xu1998hz/sescore_german_mt',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'xu1998hz/sescore_english_coco',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'xu1998hz/sescore_english_webnlg',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'unnati/kendall_tau_distance',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Viona/fuzzy_reordering',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Viona/kendall_tau',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'lhy/hamming_loss', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'lhy/ranking_loss', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'Muennighoff/code_eval_octopack',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 2},\n",
       " {'name': 'yuyijiong/quad_match_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'Splend1dchan/cosine_similarity',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'AlhitawiMohammed22/CER_Hu-Evaluation-Metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 2},\n",
       " {'name': 'Yeshwant123/mcc', 'type': 'metric', 'community': True, 'likes': 2},\n",
       " {'name': 'phonemetransformers/segmentation_scores',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'sma2023/wil', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'chanelcolgate/average_precision',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ckb/unigram', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'Felipehonorato/eer',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'manueldeprada/beer',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'shunzh/apps_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'He-Xingwei/sari_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'langdonholmes/cohen_weighted_kappa',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'fschlatt/ner_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'hyperml/balanced_accuracy',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'brian920128/doc_retrieve_metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'guydav/restrictedpython_code_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'k4black/codebleu', 'type': 'metric', 'community': True, 'likes': 1},\n",
       " {'name': 'Natooz/ece', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'ingyu/klue_mrc', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'Vipitis/shadermatch',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 8},\n",
       " {'name': 'gabeorlanski/bc_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'jjkim0807/code_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'repllabs/mean_reciprocal_rank',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'repllabs/mean_average_precision',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'mtc/fragments', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'DarrenChensformer/eval_keyphrase',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'kedudzic/charmatch',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Vallp/ter', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'DarrenChensformer/relation_extraction',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Ikala-allen/relation_extraction',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'danieldux/hierarchical_softmax_loss',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'nlpln/tst', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'bdsaglam/jer', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'davebulaval/meaningbert',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'fnvls/bleu1234', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'fnvls/bleu_1234', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'nevikw39/specificity',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'yqsong/execution_accuracy',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'shalakasatheesh/squad_v2',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'arthurvqin/pr_auc',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'd-matrix/dmx_perplexity',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'akki2825/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'juliakaczor/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'chimene/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Vickyage/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Qui-nn/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'TelEl/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'livvie/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'DaliaCaRo/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'alvinasvk/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'LottieW/accents_unplugged_eval',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'LuckiestOne/valid_efficiency_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Fritz02/execution_accuracy',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'huanghuayu/multiclass_brier_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'jialinsong/apps_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'DoctorSlimm/bangalore_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'agkphysics/ccc', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'DoctorSlimm/kaushiks_criteria',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'CZLC/rouge_raw', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'bascobasculino/mot-metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'SEA-AI/mot-metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'SEA-AI/det-metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'saicharan2804/my_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'red1bluelost/evaluate_genericify_cpp',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'maksymdolgikh/seqeval_with_fbeta',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Bekhouche/NED', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'danieldux/isco_hierarchical_accuracy',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ginic/phone_errors',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'berkatil/map', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'DarrenChensformer/action_generation',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'buelfhood/fbeta_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'danasone/ru_errant',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'helena-balabin/youden_index',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'SEA-AI/panoptic-quality',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'SEA-AI/box-metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'MathewShen/bleu', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'berkatil/mrr', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'BridgeAI-Lab/SemF1',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'SEA-AI/horizon-metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'bdsaglam/musique', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'maysonma/lingo_judge_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'dannashao/span_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Aye10032/loss_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ag2435/my_metric', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'mlcore/arxiv_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'bomjin/code_eval_octopack',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'svenwey/logmetric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'bowdbeg/matching_series',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'BridgeAI-Lab/Sem-nCG',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'bowdbeg/patch_series',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'venkatasg/gleu', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'florentgbelidji/f1',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'kbmlcoding/apps_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'jijihuny/ecqa', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'prajwall/mse', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'd-matrix/dmxMetric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'dotkaio/competition_math',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'bowdbeg/docred', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'jarod0411/aucpr', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'Ruchin/jaccard_similarity',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'phucdev/blanc_score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'NathanMad/bertscore-with-torch_dtype',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'cointegrated/blaser_2_0_qe',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ahnyeonchan/Alignment-and-Uniformity',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Baleegh/Fluency_Score',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'mdocekal/multi_label_precision_recall_accuracy_fscore',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'phucdev/vihsd', 'type': 'metric', 'community': True, 'likes': 0},\n",
       " {'name': 'argmaxinc/detailed-wer',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'SEA-AI/user-friendly-metrics',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'hage2000/code_eval_stdio',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'hage2000/my_metric',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Natooz/levenshtein',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Khaliq88/execution_accuracy',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'pico-lm/perplexity',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'mtzig/cross_entropy_loss',\n",
       "  'type': 'metric',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ncoop57/levenshtein_distance',\n",
       "  'type': 'comparison',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'kaleidophon/almost_stochastic_order',\n",
       "  'type': 'comparison',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'NeuraFusionAI/Arabic-Evaluation',\n",
       "  'type': 'comparison',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'lvwerra/element_count',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'prb977/cooccurrence_count',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'NimaBoscarino/pseudo_perplexity',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'ybelkada/toxicity',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'ronaldahmed/ccl_win',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'christopher/tokens_per_byte',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'lsy641/distinct',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 1},\n",
       " {'name': 'grepLeigh/perplexity',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Charles95/element_count',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Charles95/accuracy',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0},\n",
       " {'name': 'Lucky28/honest',\n",
       "  'type': 'measurement',\n",
       "  'community': True,\n",
       "  'likes': 0}]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "加载评估函数",
   "id": "97fc1a7073fd5cb8"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-12T03:57:07.375161Z",
     "start_time": "2024-12-12T03:57:07.209409Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 本地加载评估函数\n",
    "accuracy = evaluate.load(\"./metrics/accuracy\")\n",
    "# print(accuracy)\n",
    "# 查看函数说明\n",
    "print(accuracy.description)"
   ],
   "id": "6196ca047298ab73",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with:\n",
      "Accuracy = (TP + TN) / (TP + TN + FP + FN)\n",
      " Where:\n",
      "TP: True positive\n",
      "TN: True negative\n",
      "FP: False positive\n",
      "FN: False negative\n",
      "\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "print(accuracy.inputs_description)",
   "id": "aa865d097df85b83",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-12T03:56:18.975323Z",
     "start_time": "2024-12-12T03:56:18.767369Z"
    }
   },
   "cell_type": "code",
   "source": [
    "f1 = evaluate.load(\"./metrics/f1\")\n",
    "f1"
   ],
   "id": "5d6cefbec3534ab1",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EvaluationModule(name: \"f1\", module_type: \"metric\", features: {'predictions': Value(dtype='int32', id=None), 'references': Value(dtype='int32', id=None)}, usage: \"\"\"\n",
       "Args:\n",
       "    predictions (`list` of `int`): Predicted labels.\n",
       "    references (`list` of `int`): Ground truth labels.\n",
       "    labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`, and the order of the labels if `average` is `None`. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None.\n",
       "    pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1.\n",
       "    average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`.\n",
       "\n",
       "        - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary.\n",
       "        - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives.\n",
       "        - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account.\n",
       "        - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall.\n",
       "        - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification).\n",
       "    sample_weight (`list` of `float`): Sample weights Defaults to None.\n",
       "\n",
       "Returns:\n",
       "    f1 (`float` or `array` of `float`): F1 score or list of f1 scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher f1 scores are better.\n",
       "\n",
       "Examples:\n",
       "\n",
       "    Example 1-A simple binary example\n",
       "        >>> f1_metric = evaluate.load(\"f1\")\n",
       "        >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0])\n",
       "        >>> print(results)\n",
       "        {'f1': 0.5}\n",
       "\n",
       "    Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`.\n",
       "        >>> f1_metric = evaluate.load(\"f1\")\n",
       "        >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0)\n",
       "        >>> print(round(results['f1'], 2))\n",
       "        0.67\n",
       "\n",
       "    Example 3-The same simple binary example as in Example 1, but with `sample_weight` included.\n",
       "        >>> f1_metric = evaluate.load(\"f1\")\n",
       "        >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3])\n",
       "        >>> print(round(results['f1'], 2))\n",
       "        0.35\n",
       "\n",
       "    Example 4-A multiclass example, with different values for the `average` input.\n",
       "        >>> predictions = [0, 2, 1, 0, 0, 1]\n",
       "        >>> references = [0, 1, 2, 0, 1, 2]\n",
       "        >>> results = f1_metric.compute(predictions=predictions, references=references, average=\"macro\")\n",
       "        >>> print(round(results['f1'], 2))\n",
       "        0.27\n",
       "        >>> results = f1_metric.compute(predictions=predictions, references=references, average=\"micro\")\n",
       "        >>> print(round(results['f1'], 2))\n",
       "        0.33\n",
       "        >>> results = f1_metric.compute(predictions=predictions, references=references, average=\"weighted\")\n",
       "        >>> print(round(results['f1'], 2))\n",
       "        0.27\n",
       "        >>> results = f1_metric.compute(predictions=predictions, references=references, average=None)\n",
       "        >>> print(results)\n",
       "        {'f1': array([0.8, 0. , 0. ])}\n",
       "\n",
       "    Example 5-A multi-label example\n",
       "        >>> f1_metric = evaluate.load(\"f1\", \"multilabel\")\n",
       "        >>> results = f1_metric.compute(predictions=[[0, 1, 1], [1, 1, 0]], references=[[0, 1, 1], [0, 1, 0]], average=\"macro\")\n",
       "        >>> print(round(results['f1'], 2))\n",
       "        0.67\n",
       "\"\"\", stored examples: 0)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "评估指标计算——全局计算",
   "id": "4edeb15319a188e4"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-12T03:57:16.586814Z",
     "start_time": "2024-12-12T03:57:16.545131Z"
    }
   },
   "cell_type": "code",
   "source": [
    "results = accuracy.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])\n",
    "results"
   ],
   "id": "30335b8a46222baa",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.5}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-12T03:56:55.954311Z",
     "start_time": "2024-12-12T03:56:55.857519Z"
    }
   },
   "cell_type": "code",
   "source": "f1_results = f1.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], average=\"macro\")\n",
   "id": "a9c5ff9999dbeaa2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.43333333333333335}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "评估指标计算——迭代计算",
   "id": "cb9c35477760e2b2"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "for ref, pred in zip([0, 1, 0, 1], [1, 0, 0, 1]):\n",
    "    print(ref, pred)\n",
    "    accuracy.add(references=ref, predictions=pred)\n",
    "accuracy.compute()"
   ],
   "id": "f52d9468863d6a2c",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "多指标评估",
   "id": "8a4566562e57101"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-12T03:58:57.749232Z",
     "start_time": "2024-12-12T03:58:57.425463Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "clf_metrics = evaluate.combine([\"./metrics/accuracy\", \"./metrics/f1\"])\n",
    "clf_metrics"
   ],
   "id": "723b3054f301d2a2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<evaluate.module.CombinedEvaluations at 0x21ac9f92790>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-12T03:59:00.163381Z",
     "start_time": "2024-12-12T03:59:00.121745Z"
    }
   },
   "cell_type": "code",
   "source": "clf_metrics.compute(predictions=[0, 1, 0], references=[0, 1, 1])",
   "id": "a14744ed4bce1f97",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.6666666666666666, 'f1': 0.6666666666666666}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
